{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_args = {}\n",
    "dataset_args[\"keep_linebreaks\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-706b6e6ca36d15ce\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset text/default to /home/husein/.cache/huggingface/datasets/text/default-706b6e6ca36d15ce/0.0.0/4b86d314f7236db91f0a0f5cda32d4375445e64c5eda2692655dd99c2dac68e8...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a579986d5d944a29b8026f5a0ab38e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3a37c58ab8d465582bf7f2f382e6020",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset text downloaded and prepared to /home/husein/.cache/huggingface/datasets/text/default-706b6e6ca36d15ce/0.0.0/4b86d314f7236db91f0a0f5cda32d4375445e64c5eda2692655dd99c2dac68e8. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6524b66396ee4d04967db2f931d88912",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_files = 'sample-wiki.txt'\n",
    "extension = 'text'\n",
    "raw_datasets = load_dataset(\n",
    "    extension,\n",
    "    data_files=data_files,\n",
    "    **dataset_args,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text'],\n",
       "        num_rows: 10000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['text']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets[\"train\"].column_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names = raw_datasets[\"train\"].column_names\n",
    "text_column_name = \"text\" if \"text\" in column_names else column_names[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers.testing_utils import CaptureLogger\n",
    "from transformers import AutoTokenizer\n",
    "import transformers\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('./malay-cased-gpt2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_logger = transformers.utils.logging.get_logger(\"transformers.tokenization_utils_base\")\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    with CaptureLogger(tok_logger) as cl:\n",
    "        output = tokenizer(examples[text_column_name])\n",
    "    # clm input could be much much longer than block_size\n",
    "    if \"Token indices sequence length is longer than the\" in cl.out:\n",
    "        tok_logger.warning(\n",
    "            \"^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits\"\n",
    "            \" before being passed to the model.\"\n",
    "        )\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "958ecd2f4f1d40048fdd284b15ffc874",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Running tokenizer on dataset:   0%|          | 0/10 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_datasets = raw_datasets.map(\n",
    "    tokenize_function,\n",
    "    batched=True,\n",
    "    remove_columns=column_names,\n",
    "    desc=\"Running tokenizer on dataset\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import chain\n",
    "\n",
    "block_size = 1024\n",
    "\n",
    "def group_texts(examples):\n",
    "    # Concatenate all texts.\n",
    "    concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}\n",
    "    total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
    "    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n",
    "    # customize this part to your needs.\n",
    "    if total_length >= block_size:\n",
    "        total_length = (total_length // block_size) * block_size\n",
    "    # Split by chunks of max_len.\n",
    "    result = {\n",
    "        k: [t[i : i + block_size] for i in range(0, total_length, block_size) if len(t[i : i + block_size]) == block_size]\n",
    "        for k, t in concatenated_examples.items()\n",
    "    }\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8315dc4742e4039aed8fa9b845d7603",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Grouping texts in chunks of 1024:   0%|          | 0/10 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lm_datasets = tokenized_datasets.map(\n",
    "    group_texts,\n",
    "    batched=True,\n",
    "    desc=f\"Grouping texts in chunks of {block_size}\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[202,\n",
       " 17724,\n",
       " 78,\n",
       " 2459,\n",
       " 339,\n",
       " 264,\n",
       " 277,\n",
       " 450,\n",
       " 46,\n",
       " 264,\n",
       " 277,\n",
       " 16,\n",
       " 20319,\n",
       " 5,\n",
       " 440,\n",
       " 879,\n",
       " 312,\n",
       " 269,\n",
       " 354,\n",
       " 6986,\n",
       " 1731,\n",
       " 3568,\n",
       " 8288,\n",
       " 12,\n",
       " 560,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 5172,\n",
       " 292,\n",
       " 3664,\n",
       " 281,\n",
       " 8346,\n",
       " 16451,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 1550,\n",
       " 3664,\n",
       " 388,\n",
       " 5343,\n",
       " 328,\n",
       " 579,\n",
       " 91,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 43,\n",
       " 5565,\n",
       " 82,\n",
       " 16,\n",
       " 3474,\n",
       " 1172,\n",
       " 17,\n",
       " 202,\n",
       " 43,\n",
       " 5565,\n",
       " 82,\n",
       " 16,\n",
       " 3474,\n",
       " 1172,\n",
       " 354,\n",
       " 3748,\n",
       " 962,\n",
       " 2061,\n",
       " 517,\n",
       " 399,\n",
       " 673,\n",
       " 606,\n",
       " 15074,\n",
       " 9945,\n",
       " 3696,\n",
       " 281,\n",
       " 339,\n",
       " 264,\n",
       " 7274,\n",
       " 14056,\n",
       " 87,\n",
       " 15,\n",
       " 14789,\n",
       " 17,\n",
       " 202,\n",
       " 1225,\n",
       " 560,\n",
       " 15074,\n",
       " 2142,\n",
       " 14789,\n",
       " 3986,\n",
       " 81,\n",
       " 3278,\n",
       " 17,\n",
       " 202,\n",
       " 27736,\n",
       " 1541,\n",
       " 2061,\n",
       " 517,\n",
       " 4645,\n",
       " 5816,\n",
       " 72,\n",
       " 708,\n",
       " 443,\n",
       " 295,\n",
       " 15,\n",
       " 3463,\n",
       " 430,\n",
       " 9299,\n",
       " 297,\n",
       " 778,\n",
       " 10369,\n",
       " 1334,\n",
       " 1267,\n",
       " 15,\n",
       " 20080,\n",
       " 17,\n",
       " 202,\n",
       " 1225,\n",
       " 27940,\n",
       " 430,\n",
       " 3345,\n",
       " 297,\n",
       " 9747,\n",
       " 367,\n",
       " 440,\n",
       " 5565,\n",
       " 82,\n",
       " 16,\n",
       " 3474,\n",
       " 1172,\n",
       " 292,\n",
       " 910,\n",
       " 15,\n",
       " 502,\n",
       " 2061,\n",
       " 1181,\n",
       " 2121,\n",
       " 2423,\n",
       " 3051,\n",
       " 367,\n",
       " 1195,\n",
       " 450,\n",
       " 58,\n",
       " 299,\n",
       " 5816,\n",
       " 72,\n",
       " 708,\n",
       " 443,\n",
       " 295,\n",
       " 1073,\n",
       " 202,\n",
       " 1225,\n",
       " 486,\n",
       " 1128,\n",
       " 565,\n",
       " 536,\n",
       " 1217,\n",
       " 15074,\n",
       " 388,\n",
       " 15641,\n",
       " 20578,\n",
       " 2791,\n",
       " 15,\n",
       " 297,\n",
       " 3463,\n",
       " 388,\n",
       " 7943,\n",
       " 6531,\n",
       " 15,\n",
       " 1039,\n",
       " 9326,\n",
       " 17,\n",
       " 202,\n",
       " 3337,\n",
       " 5290,\n",
       " 652,\n",
       " 15,\n",
       " 10369,\n",
       " 15074,\n",
       " 376,\n",
       " 1622,\n",
       " 14198,\n",
       " 388,\n",
       " 11005,\n",
       " 445,\n",
       " 1731,\n",
       " 15,\n",
       " 6253,\n",
       " 17,\n",
       " 202,\n",
       " 27736,\n",
       " 376,\n",
       " 5783,\n",
       " 668,\n",
       " 3279,\n",
       " 430,\n",
       " 1079,\n",
       " 3287,\n",
       " 3051,\n",
       " 367,\n",
       " 608,\n",
       " 2972,\n",
       " 874,\n",
       " 388,\n",
       " 4335,\n",
       " 12732,\n",
       " 281,\n",
       " 753,\n",
       " 14789,\n",
       " 297,\n",
       " 2665,\n",
       " 281,\n",
       " 2922,\n",
       " 3594,\n",
       " 15,\n",
       " 6253,\n",
       " 9326,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 939,\n",
       " 2757,\n",
       " 993,\n",
       " 7408,\n",
       " 13688,\n",
       " 9246,\n",
       " 354,\n",
       " 6986,\n",
       " 538,\n",
       " 3946,\n",
       " 7329,\n",
       " 12,\n",
       " 292,\n",
       " 3376,\n",
       " 2061,\n",
       " 517,\n",
       " 2368,\n",
       " 420,\n",
       " 560,\n",
       " 824,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 9243,\n",
       " 292,\n",
       " 3664,\n",
       " 358,\n",
       " 8346,\n",
       " 16451,\n",
       " 388,\n",
       " 5343,\n",
       " 6107,\n",
       " 10158,\n",
       " 11728,\n",
       " 5371,\n",
       " 2987,\n",
       " 15,\n",
       " 3851,\n",
       " 2231,\n",
       " 82,\n",
       " 1132,\n",
       " 1054,\n",
       " 297,\n",
       " 502,\n",
       " 2142,\n",
       " 3386,\n",
       " 3964,\n",
       " 2620,\n",
       " 1557,\n",
       " 9243,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 4584,\n",
       " 5765,\n",
       " 1334,\n",
       " 2250,\n",
       " 1449,\n",
       " 292,\n",
       " 944,\n",
       " 9089,\n",
       " 281,\n",
       " 8014,\n",
       " 5420,\n",
       " 3301,\n",
       " 10467,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 502,\n",
       " 486,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 1100,\n",
       " 404,\n",
       " 1134,\n",
       " 16,\n",
       " 2083,\n",
       " 7927,\n",
       " 292,\n",
       " 1741,\n",
       " 9750,\n",
       " 1449,\n",
       " 358,\n",
       " 2184,\n",
       " 9445,\n",
       " 5420,\n",
       " 3301,\n",
       " 15,\n",
       " 297,\n",
       " 3015,\n",
       " 376,\n",
       " 502,\n",
       " 489,\n",
       " 825,\n",
       " 24811,\n",
       " 477,\n",
       " 12122,\n",
       " 1274,\n",
       " 323,\n",
       " 977,\n",
       " 264,\n",
       " 332,\n",
       " 430,\n",
       " 528,\n",
       " 2371,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 2456,\n",
       " 2560,\n",
       " 517,\n",
       " 2810,\n",
       " 1557,\n",
       " 4278,\n",
       " 3358,\n",
       " 2577,\n",
       " 748,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 489,\n",
       " 22520,\n",
       " 517,\n",
       " 26934,\n",
       " 8163,\n",
       " 3301,\n",
       " 10467,\n",
       " 1334,\n",
       " 3321,\n",
       " 1277,\n",
       " 15,\n",
       " 2184,\n",
       " 8398,\n",
       " 12287,\n",
       " 16,\n",
       " 12054,\n",
       " 354,\n",
       " 4690,\n",
       " 528,\n",
       " 3137,\n",
       " 1310,\n",
       " 1865,\n",
       " 12,\n",
       " 297,\n",
       " 6716,\n",
       " 9125,\n",
       " 1242,\n",
       " 4489,\n",
       " 744,\n",
       " 430,\n",
       " 528,\n",
       " 2334,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 486,\n",
       " 1431,\n",
       " 2142,\n",
       " 2620,\n",
       " 9243,\n",
       " 292,\n",
       " 4939,\n",
       " 12489,\n",
       " 8874,\n",
       " 430,\n",
       " 528,\n",
       " 3297,\n",
       " 297,\n",
       " 4865,\n",
       " 11288,\n",
       " 14622,\n",
       " 20213,\n",
       " 2791,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 502,\n",
       " 12181,\n",
       " 10158,\n",
       " 14475,\n",
       " 430,\n",
       " 5098,\n",
       " 88,\n",
       " 2026,\n",
       " 3301,\n",
       " 5371,\n",
       " 10467,\n",
       " 3297,\n",
       " 297,\n",
       " 4939,\n",
       " 1134,\n",
       " 16,\n",
       " 2083,\n",
       " 6716,\n",
       " 10158,\n",
       " 14475,\n",
       " 517,\n",
       " 2810,\n",
       " 2744,\n",
       " 297,\n",
       " 9350,\n",
       " 12303,\n",
       " 14475,\n",
       " 517,\n",
       " 1482,\n",
       " 4373,\n",
       " 16430,\n",
       " 358,\n",
       " 5420,\n",
       " 3301,\n",
       " 5371,\n",
       " 3051,\n",
       " 2176,\n",
       " 9758,\n",
       " 9243,\n",
       " 293,\n",
       " 4335,\n",
       " 1327,\n",
       " 8014,\n",
       " 17,\n",
       " 202,\n",
       " 900,\n",
       " 2936,\n",
       " 2566,\n",
       " 15,\n",
       " 802,\n",
       " 22520,\n",
       " 517,\n",
       " 1128,\n",
       " 565,\n",
       " 404,\n",
       " 3321,\n",
       " 18524,\n",
       " 5420,\n",
       " 3301,\n",
       " 10467,\n",
       " 1914,\n",
       " 281,\n",
       " 9243,\n",
       " 15,\n",
       " 1174,\n",
       " 18713,\n",
       " 2966,\n",
       " 82,\n",
       " 15,\n",
       " 364,\n",
       " 378,\n",
       " 17535,\n",
       " 15,\n",
       " 5614,\n",
       " 11268,\n",
       " 20532,\n",
       " 82,\n",
       " 14626,\n",
       " 1950,\n",
       " 15,\n",
       " 15427,\n",
       " 82,\n",
       " 15,\n",
       " 297,\n",
       " 311,\n",
       " 7905,\n",
       " 925,\n",
       " 316,\n",
       " 1534,\n",
       " 82,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 502,\n",
       " 3347,\n",
       " 358,\n",
       " 4281,\n",
       " 4251,\n",
       " 26081,\n",
       " 6150,\n",
       " 741,\n",
       " 450,\n",
       " 869,\n",
       " 26995,\n",
       " 29093,\n",
       " 5,\n",
       " 13007,\n",
       " 323,\n",
       " 89,\n",
       " 417,\n",
       " 9769,\n",
       " 12770,\n",
       " 17196,\n",
       " 404,\n",
       " 528,\n",
       " 2566,\n",
       " 17,\n",
       " 202,\n",
       " 900,\n",
       " 1322,\n",
       " 3568,\n",
       " 2608,\n",
       " 15,\n",
       " 2368,\n",
       " 420,\n",
       " 486,\n",
       " 1128,\n",
       " 565,\n",
       " 536,\n",
       " 5651,\n",
       " 497,\n",
       " 292,\n",
       " 1761,\n",
       " 29536,\n",
       " 11288,\n",
       " 358,\n",
       " 4865,\n",
       " 11288,\n",
       " 281,\n",
       " 16322,\n",
       " 17,\n",
       " 202,\n",
       " 37,\n",
       " 686,\n",
       " 2368,\n",
       " 420,\n",
       " 4471,\n",
       " 477,\n",
       " 11936,\n",
       " 1557,\n",
       " 9243,\n",
       " 13468,\n",
       " 1172,\n",
       " 313,\n",
       " 981,\n",
       " 490,\n",
       " 82,\n",
       " 1040,\n",
       " 802,\n",
       " 3425,\n",
       " 2053,\n",
       " 528,\n",
       " 17,\n",
       " 202,\n",
       " 3695,\n",
       " 3664,\n",
       " 388,\n",
       " 2142,\n",
       " 8870,\n",
       " 452,\n",
       " 9003,\n",
       " 15,\n",
       " 2368,\n",
       " 420,\n",
       " 4234,\n",
       " 16287,\n",
       " 3922,\n",
       " 356,\n",
       " 281,\n",
       " 939,\n",
       " 277,\n",
       " 4246,\n",
       " 7408,\n",
       " 525,\n",
       " 395,\n",
       " 430,\n",
       " 528,\n",
       " 3476,\n",
       " 17,\n",
       " 202,\n",
       " 3337,\n",
       " 846,\n",
       " 528,\n",
       " 15,\n",
       " 802,\n",
       " 9980,\n",
       " 293,\n",
       " 26505,\n",
       " 10964,\n",
       " 1809,\n",
       " 741,\n",
       " 606,\n",
       " 5343,\n",
       " 1553,\n",
       " 281,\n",
       " 994,\n",
       " 3028,\n",
       " 277,\n",
       " 525,\n",
       " 794,\n",
       " 298,\n",
       " 15,\n",
       " 281,\n",
       " 738,\n",
       " 802,\n",
       " 484,\n",
       " 3664,\n",
       " 1847,\n",
       " 846,\n",
       " 3161,\n",
       " 997,\n",
       " 3785,\n",
       " 5343,\n",
       " 505,\n",
       " 3033,\n",
       " 68,\n",
       " 22669,\n",
       " 404,\n",
       " 14708,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 56,\n",
       " 3033,\n",
       " 68,\n",
       " 1368,\n",
       " 293,\n",
       " 1327,\n",
       " 5420,\n",
       " 15641,\n",
       " 15,\n",
       " 4939,\n",
       " 1217,\n",
       " 16,\n",
       " 19,\n",
       " 3832,\n",
       " 16801,\n",
       " 14056,\n",
       " 87,\n",
       " 430,\n",
       " 21883,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 491,\n",
       " 4373,\n",
       " 1290,\n",
       " 1449,\n",
       " 1300,\n",
       " 846,\n",
       " 1500,\n",
       " 4431,\n",
       " 17,\n",
       " 202,\n",
       " 1120,\n",
       " 10669,\n",
       " 15,\n",
       " 3040,\n",
       " 15,\n",
       " 505,\n",
       " 3033,\n",
       " 68,\n",
       " 281,\n",
       " 1095,\n",
       " 1134,\n",
       " 15,\n",
       " 24444,\n",
       " 367,\n",
       " 565,\n",
       " 1459,\n",
       " 477,\n",
       " 394,\n",
       " 6031,\n",
       " 74,\n",
       " 22390,\n",
       " 17,\n",
       " 202,\n",
       " 56,\n",
       " 3033,\n",
       " 68,\n",
       " 8479,\n",
       " 4944,\n",
       " 22694,\n",
       " 1449,\n",
       " 358,\n",
       " 10669,\n",
       " 15,\n",
       " 6531,\n",
       " 587,\n",
       " 536,\n",
       " 2142,\n",
       " 9254,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 491,\n",
       " 4373,\n",
       " 3247,\n",
       " 1449,\n",
       " 10669,\n",
       " 15,\n",
       " 297,\n",
       " 502,\n",
       " 281,\n",
       " 5325,\n",
       " 1482,\n",
       " 4373,\n",
       " 430,\n",
       " 1327,\n",
       " 26254,\n",
       " 3832,\n",
       " 394,\n",
       " 6031,\n",
       " 74,\n",
       " 22390,\n",
       " 15,\n",
       " 292,\n",
       " 505,\n",
       " 3033,\n",
       " 68,\n",
       " 6928,\n",
       " 15,\n",
       " 392,\n",
       " 384,\n",
       " 17,\n",
       " 202,\n",
       " 900,\n",
       " 1322,\n",
       " 2612,\n",
       " 3051,\n",
       " 15,\n",
       " 2368,\n",
       " 420,\n",
       " 3570,\n",
       " 6838,\n",
       " 15641,\n",
       " 5420,\n",
       " 14011,\n",
       " 388,\n",
       " 1277,\n",
       " 2551,\n",
       " 1113,\n",
       " 505,\n",
       " 3033,\n",
       " 68,\n",
       " 6928,\n",
       " 384,\n",
       " 16,\n",
       " 19,\n",
       " 430,\n",
       " 21883,\n",
       " 445,\n",
       " 30812,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 2764,\n",
       " 489,\n",
       " 3775,\n",
       " 565,\n",
       " 24724,\n",
       " 26254,\n",
       " 2620,\n",
       " 14708,\n",
       " 388,\n",
       " 17993,\n",
       " 356,\n",
       " 2176,\n",
       " 12637,\n",
       " 328,\n",
       " 3101,\n",
       " 678,\n",
       " 16,\n",
       " 22,\n",
       " 358,\n",
       " 565,\n",
       " 4335,\n",
       " 281,\n",
       " 738,\n",
       " 463,\n",
       " 9077,\n",
       " 5896,\n",
       " 16,\n",
       " 520,\n",
       " 3663,\n",
       " 15,\n",
       " 292,\n",
       " 3180,\n",
       " 608,\n",
       " 1449,\n",
       " 6103,\n",
       " 1290,\n",
       " 4092,\n",
       " 997,\n",
       " 748,\n",
       " 17,\n",
       " 202,\n",
       " 56,\n",
       " 3033,\n",
       " 68,\n",
       " 4939,\n",
       " 10669,\n",
       " 15,\n",
       " 6528,\n",
       " 1459,\n",
       " 16575,\n",
       " 394,\n",
       " 6031,\n",
       " 74,\n",
       " 22390,\n",
       " 15,\n",
       " 297,\n",
       " 2889,\n",
       " 1449,\n",
       " 2368,\n",
       " 420,\n",
       " 1891,\n",
       " 565,\n",
       " 1449,\n",
       " 2027,\n",
       " 463,\n",
       " 281,\n",
       " 2404,\n",
       " 1482,\n",
       " 4373,\n",
       " 16430,\n",
       " 17760,\n",
       " 420,\n",
       " 3914,\n",
       " 8845,\n",
       " 17,\n",
       " 202,\n",
       " 22861,\n",
       " 2368,\n",
       " 420,\n",
       " 297,\n",
       " 17144,\n",
       " 3386,\n",
       " 3964,\n",
       " 9810,\n",
       " 281,\n",
       " 358,\n",
       " 26081,\n",
       " 6150,\n",
       " 14708,\n",
       " 3345,\n",
       " 450,\n",
       " 6193,\n",
       " 420,\n",
       " 552,\n",
       " 313,\n",
       " 1523,\n",
       " 18959,\n",
       " 273,\n",
       " 5,\n",
       " 2606,\n",
       " 6193,\n",
       " 420,\n",
       " 15,\n",
       " 341,\n",
       " 27683,\n",
       " 410,\n",
       " 27491,\n",
       " 9738,\n",
       " 202,\n",
       " 202,\n",
       " 202,\n",
       " 4714,\n",
       " 613,\n",
       " 68,\n",
       " 28962,\n",
       " 2547,\n",
       " 15,\n",
       " 452,\n",
       " 27114,\n",
       " 837,\n",
       " 505,\n",
       " 988,\n",
       " 9113,\n",
       " 72,\n",
       " 614,\n",
       " 48,\n",
       " 4705,\n",
       " 15,\n",
       " 354,\n",
       " 6986,\n",
       " 538,\n",
       " 2664,\n",
       " 15,\n",
       " 8928,\n",
       " 281,\n",
       " 537,\n",
       " 6033,\n",
       " 82,\n",
       " 15,\n",
       " 24185,\n",
       " 1119,\n",
       " 560,\n",
       " 824,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 4630,\n",
       " 292,\n",
       " 24156,\n",
       " 5420,\n",
       " 3301,\n",
       " 297,\n",
       " 3664,\n",
       " 388,\n",
       " 5343,\n",
       " 328,\n",
       " 17,\n",
       " 38,\n",
       " 17,\n",
       " 18746,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 202,\n",
       " 24439,\n",
       " 630,\n",
       " 6157,\n",
       " 4530,\n",
       " 411,\n",
       " 15,\n",
       " 452,\n",
       " 27114,\n",
       " 837,\n",
       " 505,\n",
       " 988,\n",
       " 9113,\n",
       " 72,\n",
       " 614,\n",
       " 48,\n",
       " 4705,\n",
       " 15,\n",
       " 354,\n",
       " 4784,\n",
       " 3626,\n",
       " 8288,\n",
       " 12,\n",
       " 560,\n",
       " 824,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 4630,\n",
       " 15,\n",
       " 292,\n",
       " 1622,\n",
       " 3664,\n",
       " 388,\n",
       " 5343,\n",
       " 328,\n",
       " 17,\n",
       " 54,\n",
       " 17,\n",
       " 29585,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 560,\n",
       " 2810,\n",
       " 2113,\n",
       " 292,\n",
       " 10170,\n",
       " 297,\n",
       " 824,\n",
       " 539,\n",
       " 345,\n",
       " 309,\n",
       " 292,\n",
       " 2466,\n",
       " 367,\n",
       " 496,\n",
       " 4681,\n",
       " 388,\n",
       " 9750,\n",
       " 1449,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 28775,\n",
       " ...]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_datasets['train']['input_ids'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[202,\n",
       " 17724,\n",
       " 78,\n",
       " 2459,\n",
       " 339,\n",
       " 264,\n",
       " 277,\n",
       " 450,\n",
       " 46,\n",
       " 264,\n",
       " 277,\n",
       " 16,\n",
       " 20319,\n",
       " 5,\n",
       " 440,\n",
       " 879,\n",
       " 312,\n",
       " 269,\n",
       " 354,\n",
       " 6986,\n",
       " 1731,\n",
       " 3568,\n",
       " 8288,\n",
       " 12,\n",
       " 560,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 5172,\n",
       " 292,\n",
       " 3664,\n",
       " 281,\n",
       " 8346,\n",
       " 16451,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 1550,\n",
       " 3664,\n",
       " 388,\n",
       " 5343,\n",
       " 328,\n",
       " 579,\n",
       " 91,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 43,\n",
       " 5565,\n",
       " 82,\n",
       " 16,\n",
       " 3474,\n",
       " 1172,\n",
       " 17,\n",
       " 202,\n",
       " 43,\n",
       " 5565,\n",
       " 82,\n",
       " 16,\n",
       " 3474,\n",
       " 1172,\n",
       " 354,\n",
       " 3748,\n",
       " 962,\n",
       " 2061,\n",
       " 517,\n",
       " 399,\n",
       " 673,\n",
       " 606,\n",
       " 15074,\n",
       " 9945,\n",
       " 3696,\n",
       " 281,\n",
       " 339,\n",
       " 264,\n",
       " 7274,\n",
       " 14056,\n",
       " 87,\n",
       " 15,\n",
       " 14789,\n",
       " 17,\n",
       " 202,\n",
       " 1225,\n",
       " 560,\n",
       " 15074,\n",
       " 2142,\n",
       " 14789,\n",
       " 3986,\n",
       " 81,\n",
       " 3278,\n",
       " 17,\n",
       " 202,\n",
       " 27736,\n",
       " 1541,\n",
       " 2061,\n",
       " 517,\n",
       " 4645,\n",
       " 5816,\n",
       " 72,\n",
       " 708,\n",
       " 443,\n",
       " 295,\n",
       " 15,\n",
       " 3463,\n",
       " 430,\n",
       " 9299,\n",
       " 297,\n",
       " 778,\n",
       " 10369,\n",
       " 1334,\n",
       " 1267,\n",
       " 15,\n",
       " 20080,\n",
       " 17,\n",
       " 202,\n",
       " 1225,\n",
       " 27940,\n",
       " 430,\n",
       " 3345,\n",
       " 297,\n",
       " 9747,\n",
       " 367,\n",
       " 440,\n",
       " 5565,\n",
       " 82,\n",
       " 16,\n",
       " 3474,\n",
       " 1172,\n",
       " 292,\n",
       " 910,\n",
       " 15,\n",
       " 502,\n",
       " 2061,\n",
       " 1181,\n",
       " 2121,\n",
       " 2423,\n",
       " 3051,\n",
       " 367,\n",
       " 1195,\n",
       " 450,\n",
       " 58,\n",
       " 299,\n",
       " 5816,\n",
       " 72,\n",
       " 708,\n",
       " 443,\n",
       " 295,\n",
       " 1073,\n",
       " 202,\n",
       " 1225,\n",
       " 486,\n",
       " 1128,\n",
       " 565,\n",
       " 536,\n",
       " 1217,\n",
       " 15074,\n",
       " 388,\n",
       " 15641,\n",
       " 20578,\n",
       " 2791,\n",
       " 15,\n",
       " 297,\n",
       " 3463,\n",
       " 388,\n",
       " 7943,\n",
       " 6531,\n",
       " 15,\n",
       " 1039,\n",
       " 9326,\n",
       " 17,\n",
       " 202,\n",
       " 3337,\n",
       " 5290,\n",
       " 652,\n",
       " 15,\n",
       " 10369,\n",
       " 15074,\n",
       " 376,\n",
       " 1622,\n",
       " 14198,\n",
       " 388,\n",
       " 11005,\n",
       " 445,\n",
       " 1731,\n",
       " 15,\n",
       " 6253,\n",
       " 17,\n",
       " 202,\n",
       " 27736,\n",
       " 376,\n",
       " 5783,\n",
       " 668,\n",
       " 3279,\n",
       " 430,\n",
       " 1079,\n",
       " 3287,\n",
       " 3051,\n",
       " 367,\n",
       " 608,\n",
       " 2972,\n",
       " 874,\n",
       " 388,\n",
       " 4335,\n",
       " 12732,\n",
       " 281,\n",
       " 753,\n",
       " 14789,\n",
       " 297,\n",
       " 2665,\n",
       " 281,\n",
       " 2922,\n",
       " 3594,\n",
       " 15,\n",
       " 6253,\n",
       " 9326,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 939,\n",
       " 2757,\n",
       " 993,\n",
       " 7408,\n",
       " 13688,\n",
       " 9246,\n",
       " 354,\n",
       " 6986,\n",
       " 538,\n",
       " 3946,\n",
       " 7329,\n",
       " 12,\n",
       " 292,\n",
       " 3376,\n",
       " 2061,\n",
       " 517,\n",
       " 2368,\n",
       " 420,\n",
       " 560,\n",
       " 824,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 9243,\n",
       " 292,\n",
       " 3664,\n",
       " 358,\n",
       " 8346,\n",
       " 16451,\n",
       " 388,\n",
       " 5343,\n",
       " 6107,\n",
       " 10158,\n",
       " 11728,\n",
       " 5371,\n",
       " 2987,\n",
       " 15,\n",
       " 3851,\n",
       " 2231,\n",
       " 82,\n",
       " 1132,\n",
       " 1054,\n",
       " 297,\n",
       " 502,\n",
       " 2142,\n",
       " 3386,\n",
       " 3964,\n",
       " 2620,\n",
       " 1557,\n",
       " 9243,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 4584,\n",
       " 5765,\n",
       " 1334,\n",
       " 2250,\n",
       " 1449,\n",
       " 292,\n",
       " 944,\n",
       " 9089,\n",
       " 281,\n",
       " 8014,\n",
       " 5420,\n",
       " 3301,\n",
       " 10467,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 502,\n",
       " 486,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 1100,\n",
       " 404,\n",
       " 1134,\n",
       " 16,\n",
       " 2083,\n",
       " 7927,\n",
       " 292,\n",
       " 1741,\n",
       " 9750,\n",
       " 1449,\n",
       " 358,\n",
       " 2184,\n",
       " 9445,\n",
       " 5420,\n",
       " 3301,\n",
       " 15,\n",
       " 297,\n",
       " 3015,\n",
       " 376,\n",
       " 502,\n",
       " 489,\n",
       " 825,\n",
       " 24811,\n",
       " 477,\n",
       " 12122,\n",
       " 1274,\n",
       " 323,\n",
       " 977,\n",
       " 264,\n",
       " 332,\n",
       " 430,\n",
       " 528,\n",
       " 2371,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 2456,\n",
       " 2560,\n",
       " 517,\n",
       " 2810,\n",
       " 1557,\n",
       " 4278,\n",
       " 3358,\n",
       " 2577,\n",
       " 748,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 489,\n",
       " 22520,\n",
       " 517,\n",
       " 26934,\n",
       " 8163,\n",
       " 3301,\n",
       " 10467,\n",
       " 1334,\n",
       " 3321,\n",
       " 1277,\n",
       " 15,\n",
       " 2184,\n",
       " 8398,\n",
       " 12287,\n",
       " 16,\n",
       " 12054,\n",
       " 354,\n",
       " 4690,\n",
       " 528,\n",
       " 3137,\n",
       " 1310,\n",
       " 1865,\n",
       " 12,\n",
       " 297,\n",
       " 6716,\n",
       " 9125,\n",
       " 1242,\n",
       " 4489,\n",
       " 744,\n",
       " 430,\n",
       " 528,\n",
       " 2334,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 486,\n",
       " 1431,\n",
       " 2142,\n",
       " 2620,\n",
       " 9243,\n",
       " 292,\n",
       " 4939,\n",
       " 12489,\n",
       " 8874,\n",
       " 430,\n",
       " 528,\n",
       " 3297,\n",
       " 297,\n",
       " 4865,\n",
       " 11288,\n",
       " 14622,\n",
       " 20213,\n",
       " 2791,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 502,\n",
       " 12181,\n",
       " 10158,\n",
       " 14475,\n",
       " 430,\n",
       " 5098,\n",
       " 88,\n",
       " 2026,\n",
       " 3301,\n",
       " 5371,\n",
       " 10467,\n",
       " 3297,\n",
       " 297,\n",
       " 4939,\n",
       " 1134,\n",
       " 16,\n",
       " 2083,\n",
       " 6716,\n",
       " 10158,\n",
       " 14475,\n",
       " 517,\n",
       " 2810,\n",
       " 2744,\n",
       " 297,\n",
       " 9350,\n",
       " 12303,\n",
       " 14475,\n",
       " 517,\n",
       " 1482,\n",
       " 4373,\n",
       " 16430,\n",
       " 358,\n",
       " 5420,\n",
       " 3301,\n",
       " 5371,\n",
       " 3051,\n",
       " 2176,\n",
       " 9758,\n",
       " 9243,\n",
       " 293,\n",
       " 4335,\n",
       " 1327,\n",
       " 8014,\n",
       " 17,\n",
       " 202,\n",
       " 900,\n",
       " 2936,\n",
       " 2566,\n",
       " 15,\n",
       " 802,\n",
       " 22520,\n",
       " 517,\n",
       " 1128,\n",
       " 565,\n",
       " 404,\n",
       " 3321,\n",
       " 18524,\n",
       " 5420,\n",
       " 3301,\n",
       " 10467,\n",
       " 1914,\n",
       " 281,\n",
       " 9243,\n",
       " 15,\n",
       " 1174,\n",
       " 18713,\n",
       " 2966,\n",
       " 82,\n",
       " 15,\n",
       " 364,\n",
       " 378,\n",
       " 17535,\n",
       " 15,\n",
       " 5614,\n",
       " 11268,\n",
       " 20532,\n",
       " 82,\n",
       " 14626,\n",
       " 1950,\n",
       " 15,\n",
       " 15427,\n",
       " 82,\n",
       " 15,\n",
       " 297,\n",
       " 311,\n",
       " 7905,\n",
       " 925,\n",
       " 316,\n",
       " 1534,\n",
       " 82,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 502,\n",
       " 3347,\n",
       " 358,\n",
       " 4281,\n",
       " 4251,\n",
       " 26081,\n",
       " 6150,\n",
       " 741,\n",
       " 450,\n",
       " 869,\n",
       " 26995,\n",
       " 29093,\n",
       " 5,\n",
       " 13007,\n",
       " 323,\n",
       " 89,\n",
       " 417,\n",
       " 9769,\n",
       " 12770,\n",
       " 17196,\n",
       " 404,\n",
       " 528,\n",
       " 2566,\n",
       " 17,\n",
       " 202,\n",
       " 900,\n",
       " 1322,\n",
       " 3568,\n",
       " 2608,\n",
       " 15,\n",
       " 2368,\n",
       " 420,\n",
       " 486,\n",
       " 1128,\n",
       " 565,\n",
       " 536,\n",
       " 5651,\n",
       " 497,\n",
       " 292,\n",
       " 1761,\n",
       " 29536,\n",
       " 11288,\n",
       " 358,\n",
       " 4865,\n",
       " 11288,\n",
       " 281,\n",
       " 16322,\n",
       " 17,\n",
       " 202,\n",
       " 37,\n",
       " 686,\n",
       " 2368,\n",
       " 420,\n",
       " 4471,\n",
       " 477,\n",
       " 11936,\n",
       " 1557,\n",
       " 9243,\n",
       " 13468,\n",
       " 1172,\n",
       " 313,\n",
       " 981,\n",
       " 490,\n",
       " 82,\n",
       " 1040,\n",
       " 802,\n",
       " 3425,\n",
       " 2053,\n",
       " 528,\n",
       " 17,\n",
       " 202,\n",
       " 3695,\n",
       " 3664,\n",
       " 388,\n",
       " 2142,\n",
       " 8870,\n",
       " 452,\n",
       " 9003,\n",
       " 15,\n",
       " 2368,\n",
       " 420,\n",
       " 4234,\n",
       " 16287,\n",
       " 3922,\n",
       " 356,\n",
       " 281,\n",
       " 939,\n",
       " 277,\n",
       " 4246,\n",
       " 7408,\n",
       " 525,\n",
       " 395,\n",
       " 430,\n",
       " 528,\n",
       " 3476,\n",
       " 17,\n",
       " 202,\n",
       " 3337,\n",
       " 846,\n",
       " 528,\n",
       " 15,\n",
       " 802,\n",
       " 9980,\n",
       " 293,\n",
       " 26505,\n",
       " 10964,\n",
       " 1809,\n",
       " 741,\n",
       " 606,\n",
       " 5343,\n",
       " 1553,\n",
       " 281,\n",
       " 994,\n",
       " 3028,\n",
       " 277,\n",
       " 525,\n",
       " 794,\n",
       " 298,\n",
       " 15,\n",
       " 281,\n",
       " 738,\n",
       " 802,\n",
       " 484,\n",
       " 3664,\n",
       " 1847,\n",
       " 846,\n",
       " 3161,\n",
       " 997,\n",
       " 3785,\n",
       " 5343,\n",
       " 505,\n",
       " 3033,\n",
       " 68,\n",
       " 22669,\n",
       " 404,\n",
       " 14708,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 56,\n",
       " 3033,\n",
       " 68,\n",
       " 1368,\n",
       " 293,\n",
       " 1327,\n",
       " 5420,\n",
       " 15641,\n",
       " 15,\n",
       " 4939,\n",
       " 1217,\n",
       " 16,\n",
       " 19,\n",
       " 3832,\n",
       " 16801,\n",
       " 14056,\n",
       " 87,\n",
       " 430,\n",
       " 21883,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 491,\n",
       " 4373,\n",
       " 1290,\n",
       " 1449,\n",
       " 1300,\n",
       " 846,\n",
       " 1500,\n",
       " 4431,\n",
       " 17,\n",
       " 202,\n",
       " 1120,\n",
       " 10669,\n",
       " 15,\n",
       " 3040,\n",
       " 15,\n",
       " 505,\n",
       " 3033,\n",
       " 68,\n",
       " 281,\n",
       " 1095,\n",
       " 1134,\n",
       " 15,\n",
       " 24444,\n",
       " 367,\n",
       " 565,\n",
       " 1459,\n",
       " 477,\n",
       " 394,\n",
       " 6031,\n",
       " 74,\n",
       " 22390,\n",
       " 17,\n",
       " 202,\n",
       " 56,\n",
       " 3033,\n",
       " 68,\n",
       " 8479,\n",
       " 4944,\n",
       " 22694,\n",
       " 1449,\n",
       " 358,\n",
       " 10669,\n",
       " 15,\n",
       " 6531,\n",
       " 587,\n",
       " 536,\n",
       " 2142,\n",
       " 9254,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 491,\n",
       " 4373,\n",
       " 3247,\n",
       " 1449,\n",
       " 10669,\n",
       " 15,\n",
       " 297,\n",
       " 502,\n",
       " 281,\n",
       " 5325,\n",
       " 1482,\n",
       " 4373,\n",
       " 430,\n",
       " 1327,\n",
       " 26254,\n",
       " 3832,\n",
       " 394,\n",
       " 6031,\n",
       " 74,\n",
       " 22390,\n",
       " 15,\n",
       " 292,\n",
       " 505,\n",
       " 3033,\n",
       " 68,\n",
       " 6928,\n",
       " 15,\n",
       " 392,\n",
       " 384,\n",
       " 17,\n",
       " 202,\n",
       " 900,\n",
       " 1322,\n",
       " 2612,\n",
       " 3051,\n",
       " 15,\n",
       " 2368,\n",
       " 420,\n",
       " 3570,\n",
       " 6838,\n",
       " 15641,\n",
       " 5420,\n",
       " 14011,\n",
       " 388,\n",
       " 1277,\n",
       " 2551,\n",
       " 1113,\n",
       " 505,\n",
       " 3033,\n",
       " 68,\n",
       " 6928,\n",
       " 384,\n",
       " 16,\n",
       " 19,\n",
       " 430,\n",
       " 21883,\n",
       " 445,\n",
       " 30812,\n",
       " 17,\n",
       " 202,\n",
       " 6193,\n",
       " 420,\n",
       " 2764,\n",
       " 489,\n",
       " 3775,\n",
       " 565,\n",
       " 24724,\n",
       " 26254,\n",
       " 2620,\n",
       " 14708,\n",
       " 388,\n",
       " 17993,\n",
       " 356,\n",
       " 2176,\n",
       " 12637,\n",
       " 328,\n",
       " 3101,\n",
       " 678,\n",
       " 16,\n",
       " 22,\n",
       " 358,\n",
       " 565,\n",
       " 4335,\n",
       " 281,\n",
       " 738,\n",
       " 463,\n",
       " 9077,\n",
       " 5896,\n",
       " 16,\n",
       " 520,\n",
       " 3663,\n",
       " 15,\n",
       " 292,\n",
       " 3180,\n",
       " 608,\n",
       " 1449,\n",
       " 6103,\n",
       " 1290,\n",
       " 4092,\n",
       " 997,\n",
       " 748,\n",
       " 17,\n",
       " 202,\n",
       " 56,\n",
       " 3033,\n",
       " 68,\n",
       " 4939,\n",
       " 10669,\n",
       " 15,\n",
       " 6528,\n",
       " 1459,\n",
       " 16575,\n",
       " 394,\n",
       " 6031,\n",
       " 74,\n",
       " 22390,\n",
       " 15,\n",
       " 297,\n",
       " 2889,\n",
       " 1449,\n",
       " 2368,\n",
       " 420,\n",
       " 1891,\n",
       " 565,\n",
       " 1449,\n",
       " 2027,\n",
       " 463,\n",
       " 281,\n",
       " 2404,\n",
       " 1482,\n",
       " 4373,\n",
       " 16430,\n",
       " 17760,\n",
       " 420,\n",
       " 3914,\n",
       " 8845,\n",
       " 17,\n",
       " 202,\n",
       " 22861,\n",
       " 2368,\n",
       " 420,\n",
       " 297,\n",
       " 17144,\n",
       " 3386,\n",
       " 3964,\n",
       " 9810,\n",
       " 281,\n",
       " 358,\n",
       " 26081,\n",
       " 6150,\n",
       " 14708,\n",
       " 3345,\n",
       " 450,\n",
       " 6193,\n",
       " 420,\n",
       " 552,\n",
       " 313,\n",
       " 1523,\n",
       " 18959,\n",
       " 273,\n",
       " 5,\n",
       " 2606,\n",
       " 6193,\n",
       " 420,\n",
       " 15,\n",
       " 341,\n",
       " 27683,\n",
       " 410,\n",
       " 27491,\n",
       " 9738,\n",
       " 202,\n",
       " 202,\n",
       " 202,\n",
       " 4714,\n",
       " 613,\n",
       " 68,\n",
       " 28962,\n",
       " 2547,\n",
       " 15,\n",
       " 452,\n",
       " 27114,\n",
       " 837,\n",
       " 505,\n",
       " 988,\n",
       " 9113,\n",
       " 72,\n",
       " 614,\n",
       " 48,\n",
       " 4705,\n",
       " 15,\n",
       " 354,\n",
       " 6986,\n",
       " 538,\n",
       " 2664,\n",
       " 15,\n",
       " 8928,\n",
       " 281,\n",
       " 537,\n",
       " 6033,\n",
       " 82,\n",
       " 15,\n",
       " 24185,\n",
       " 1119,\n",
       " 560,\n",
       " 824,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 4630,\n",
       " 292,\n",
       " 24156,\n",
       " 5420,\n",
       " 3301,\n",
       " 297,\n",
       " 3664,\n",
       " 388,\n",
       " 5343,\n",
       " 328,\n",
       " 17,\n",
       " 38,\n",
       " 17,\n",
       " 18746,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 202,\n",
       " 24439,\n",
       " 630,\n",
       " 6157,\n",
       " 4530,\n",
       " 411,\n",
       " 15,\n",
       " 452,\n",
       " 27114,\n",
       " 837,\n",
       " 505,\n",
       " 988,\n",
       " 9113,\n",
       " 72,\n",
       " 614,\n",
       " 48,\n",
       " 4705,\n",
       " 15,\n",
       " 354,\n",
       " 4784,\n",
       " 3626,\n",
       " 8288,\n",
       " 12,\n",
       " 560,\n",
       " 824,\n",
       " 2810,\n",
       " 3386,\n",
       " 3964,\n",
       " 4630,\n",
       " 15,\n",
       " 292,\n",
       " 1622,\n",
       " 3664,\n",
       " 388,\n",
       " 5343,\n",
       " 328,\n",
       " 17,\n",
       " 54,\n",
       " 17,\n",
       " 29585,\n",
       " 17,\n",
       " 202,\n",
       " 1188,\n",
       " 560,\n",
       " 2810,\n",
       " 2113,\n",
       " 292,\n",
       " 10170,\n",
       " 297,\n",
       " 824,\n",
       " 539,\n",
       " 345,\n",
       " 309,\n",
       " 292,\n",
       " 2466,\n",
       " 367,\n",
       " 496,\n",
       " 4681,\n",
       " 388,\n",
       " 9750,\n",
       " 1449,\n",
       " 17,\n",
       " 202,\n",
       " 202,\n",
       " 28775,\n",
       " ...]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_datasets['train']['labels'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
